package com.jusyl.cristo.spider.smzdm;

import java.util.ArrayList;
import java.util.List;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

public class WebMagicSmzdm implements PageProcessor{

	private Site site = Site.me().setRetryTimes(3).setSleepTime(1000);
	
	public WebMagicSmzdm(){
	}

	@Override
	public void process(Page page) {
		//获取页面信息以供分析参考
		page.putField("page", page);
		
		//获取所有需要抓取的列表页
		int totalPage = 300;
		List<String> links = new ArrayList<String>();
		for(int i = 1; i < totalPage; i++) {
			links.add("http://4v3v.com/htm/Article2/" + i + ".htm");
		}
		page.addTargetRequests(links);
	}

	@Override
	public Site getSite() {
		return site;
	}
}
